import requests
from lxml import etree

url = "https://www.shicimingju.com/bookmark/sidamingzhu.html"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62"
}


response = requests.get(url, headers=headers)
res_content = response.content.decode()
print(res_content)

tree = etree.HTML(res_content)

# 抓取书名称和ur
hrefs = tree.xpath('//div[@class="list clear theme2 theme3"]/a/@href')
print(hrefs)

for href in hrefs:
    book_url = f"https://www.shicimingju.com{href}"
    print(book_url)
    book_chapter_response = requests.get(book_url, headers=headers)
    book_chapter_tree = etree.HTML(book_chapter_response.content.decode())
    mulu = book_chapter_tree.xpath("//div[@class='list']/a")
    for li in mulu:
        url = "https://shicimingju.com" + li.xpath("@href")[0]
        res = requests.get(url, headers=headers)
        tree = etree.HTML(res.content.decode())
        content = tree.xpath("//div[@class='text p_pad']/p/text()")
        print(''.join(content).strip())

 